{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fcf9afb2-9157-4323-b3ea-cde8832dbaf4",
   "metadata": {},
   "source": [
    "# 检索效果评估\n",
    "## 检索相关性评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9fd4ccba-5c19-40c8-a4b4-d5c60f7641bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_retrieval_effectiveness(questions):\n",
    "    \"\"\"\n",
    "    评估不同检索方法对各类问题的效果\n",
    "    \"\"\"\n",
    "    results = []\n",
    "    \n",
    "    for question in questions:\n",
    "        query_type = determine_query_type(question)\n",
    "        bm25_weight, vector_weight = get_dynamic_weights(query_type)\n",
    "        \n",
    "        # 获取不同检索器的结果\n",
    "        vectordb = get_vectordb()\n",
    "        documents = get_text_list_from_milvus(collection_name=\"Vmaxs\")\n",
    "        \n",
    "        # BM25检索\n",
    "        bm25_retriever = BM25Retriever.from_texts(documents)\n",
    "        bm25_docs = bm25_retriever.get_relevant_documents(question)\n",
    "        \n",
    "        # 向量检索\n",
    "        vector_retriever = vectordb.as_retriever(search_kwargs={\"k\": 10})\n",
    "        vector_docs = vector_retriever.get_relevant_documents(question)\n",
    "        \n",
    "        # 混合检索\n",
    "        ensemble_retriever = EnsembleRetriever(\n",
    "            retrievers=[bm25_retriever, vector_retriever],\n",
    "            weights=[bm25_weight, vector_weight]\n",
    "        )\n",
    "        ensemble_docs = ensemble_retriever.get_relevant_documents(question)\n",
    "        \n",
    "        # 人工评估相关性(简化版，实际应有人工标注)\n",
    "        bm25_rel = len([d for d in bm25_docs if is_relevant(d.page_content, question)])\n",
    "        vector_rel = len([d for d in vector_docs if is_relevant(d.page_content, question)])\n",
    "        ensemble_rel = len([d for d in ensemble_docs if is_relevant(d.page_content, question)])\n",
    "        \n",
    "        results.append({\n",
    "            \"question\": question,\n",
    "            \"query_type\": query_type,\n",
    "            \"bm25_relevant\": bm25_rel,\n",
    "            \"vector_relevant\": vector_rel,\n",
    "            \"ensemble_relevant\": ensemble_rel,\n",
    "            \"bm25_weight\": bm25_weight,\n",
    "            \"vector_weight\": vector_weight\n",
    "        })\n",
    "    \n",
    "    return results\n",
    "\n",
    "# 示例评估函数(实际使用时需要完善)\n",
    "def is_relevant(document_content, question):\n",
    "    \"\"\"简化版相关性判断，实际应有人工标注或更复杂的逻辑\"\"\"\n",
    "    return any(keyword in document_content for keyword in question.split()[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efa5d8c8-23fe-47a5-a5e4-b321213013bf",
   "metadata": {},
   "source": [
    "## 问答质量评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ab45c2c8-ab18-4ca7-9072-06aa7f80a08e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_answer_quality(questions, reference_answers):\n",
    "    \"\"\"\n",
    "    评估回答质量，对比参考答案\n",
    "    \"\"\"\n",
    "    evaluation_results = []\n",
    "    \n",
    "    for question, ref_answer in zip(questions, reference_answers):\n",
    "        result = get_qa_chain_with_memory(question)\n",
    "        answer = result['answer']\n",
    "        \n",
    "        # 计算相似度(简化版，实际可用更复杂的语义相似度计算)\n",
    "        similarity = calculate_similarity(answer, ref_answer)\n",
    "        \n",
    "        evaluation_results.append({\n",
    "            \"question\": question,\n",
    "            \"generated_answer\": answer,\n",
    "            \"reference_answer\": ref_answer,\n",
    "            \"similarity_score\": similarity,\n",
    "            \"query_type\": determine_query_type(question)\n",
    "        })\n",
    "    \n",
    "    return evaluation_results\n",
    "\n",
    "def calculate_similarity(answer1, answer2):\n",
    "    \"\"\"简化版相似度计算\"\"\"\n",
    "    words1 = set(answer1.lower().split())\n",
    "    words2 = set(answer2.lower().split())\n",
    "    intersection = words1.intersection(words2)\n",
    "    return len(intersection) / max(len(words1), len(words2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8de02808-5e4a-4905-a032-8384d7cad360",
   "metadata": {},
   "source": [
    "## 权重分配合理性评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "964758f7-c0fb-439f-9097-32776fef27b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_weight_allocation(questions):\n",
    "    \"\"\"\n",
    "    评估权重分配策略的合理性\n",
    "    \"\"\"\n",
    "    weight_results = []\n",
    "    \n",
    "    for question in questions:\n",
    "        query_type = determine_query_type(question)\n",
    "        bm25_weight, vector_weight = get_dynamic_weights(query_type)\n",
    "        \n",
    "        # 获取不同检索器的结果\n",
    "        vectordb = get_vectordb()\n",
    "        documents = get_text_list_from_milvus(collection_name=\"Vmaxs\")\n",
    "        \n",
    "        # 不同权重组合的检索效果\n",
    "        weight_combinations = [\n",
    "            (0.7, 0.3),  # 偏BM25\n",
    "            (0.3, 0.7),  # 偏向量\n",
    "            (0.5, 0.5),  # 平衡\n",
    "            (bm25_weight, vector_weight)  # 动态分配\n",
    "        ]\n",
    "        \n",
    "        best_relevant = 0\n",
    "        best_weights = (0, 0)\n",
    "        \n",
    "        for weights in weight_combinations:\n",
    "            bm25_retriever = BM25Retriever.from_texts(documents)\n",
    "            vector_retriever = vectordb.as_retriever(search_kwargs={\"k\": 10})\n",
    "            \n",
    "            ensemble_retriever = EnsembleRetriever(\n",
    "                retrievers=[bm25_retriever, vector_retriever],\n",
    "                weights=weights\n",
    "            )\n",
    "            \n",
    "            docs = ensemble_retriever.get_relevant_documents(question)\n",
    "            relevant = len([d for d in docs if is_relevant(d.page_content, question)])\n",
    "            \n",
    "            if relevant > best_relevant:\n",
    "                best_relevant = relevant\n",
    "                best_weights = weights\n",
    "        \n",
    "        weight_results.append({\n",
    "            \"question\": question,\n",
    "            \"query_type\": query_type,\n",
    "            \"dynamic_weights\": (bm25_weight, vector_weight),\n",
    "            \"best_weights\": best_weights,\n",
    "            \"match\": (bm25_weight, vector_weight) == best_weights\n",
    "        })\n",
    "    \n",
    "    return weight_results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3df44d58-67a9-4938-9e64-aaa48da7287b",
   "metadata": {},
   "source": [
    "## 综合评估报告生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c8ee9977-da9e-4633-b242-9deda1b41a38",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_evaluation_report(questions, reference_answers=None):\n",
    "    \"\"\"\n",
    "    生成完整的评估报告\n",
    "    \"\"\"\n",
    "    print(\"=\"*50)\n",
    "    print(\"混合检索问答系统评估报告\")\n",
    "    print(\"=\"*50)\n",
    "    \n",
    "    # 1. 检索效果评估\n",
    "    print(\"\\n1. 检索效果评估:\")\n",
    "    retrieval_results = evaluate_retrieval_effectiveness(questions)\n",
    "    for res in retrieval_results:\n",
    "        print(f\"\\n问题: {res['question']}\")\n",
    "        print(f\"类型: {res['query_type']}\")\n",
    "        print(f\"权重分配: BM25={res['bm25_weight']}, Vector={res['vector_weight']}\")\n",
    "        print(f\"BM25相关文档数: {res['bm25_relevant']}/10\")\n",
    "        print(f\"向量检索相关文档数: {res['vector_relevant']}/10\")\n",
    "        print(f\"混合检索相关文档数: {res['ensemble_relevant']}/10\")\n",
    "    \n",
    "    # 2. 问答质量评估(如果有参考答案)\n",
    "    if reference_answers:\n",
    "        print(\"\\n2. 问答质量评估:\")\n",
    "        answer_results = evaluate_answer_quality(questions, reference_answers)\n",
    "        avg_similarity = sum(res['similarity_score'] for res in answer_results) / len(answer_results)\n",
    "        \n",
    "        for res in answer_results:\n",
    "            print(f\"\\n问题: {res['question']}\")\n",
    "            print(f\"类型: {res['query_type']}\")\n",
    "            print(f\"相似度得分: {res['similarity_score']:.2f}\")\n",
    "        \n",
    "        print(f\"\\n平均相似度得分: {avg_similarity:.2f}\")\n",
    "    \n",
    "    # 3. 权重分配评估\n",
    "    print(\"\\n3. 权重分配合理性评估:\")\n",
    "    weight_results = evaluate_weight_allocation(questions)\n",
    "    correct_weights = sum(res['match'] for res in weight_results)\n",
    "    \n",
    "    for res in weight_results:\n",
    "        print(f\"\\n问题: {res['question']}\")\n",
    "        print(f\"类型: {res['query_type']}\")\n",
    "        print(f\"动态分配权重: BM25={res['dynamic_weights'][0]}, Vector={res['dynamic_weights'][1]}\")\n",
    "        print(f\"最佳权重: BM25={res['best_weights'][0]}, Vector={res['best_weights'][1]}\")\n",
    "        print(f\"匹配: {'是' if res['match'] else '否'}\")\n",
    "    \n",
    "    print(f\"\\n权重分配准确率: {correct_weights/len(questions):.2%}\")\n",
    "    \n",
    "    print(\"\\n评估完成!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19e74a45-2f38-4a5a-b938-7515216b844c",
   "metadata": {},
   "source": [
    "## 使用示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e1fe8e5-84b3-4d02-858d-c9047415cb82",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试问题和参考答案(示例)\n",
    "questions = [\n",
    "    \"什么是VMAX的上网日志业务？\",\n",
    "    \"上网日志业务包含哪些功能？\",\n",
    "    \"为什么我的VMAX设备会出现日志丢失问题？\",\n",
    "    \"如何解决VMAX日志存储空间不足的问题？\",\n",
    "    \"VMAX-S与其他型号的主要区别是什么？\"\n",
    "]\n",
    "\n",
    "reference_answers = [\n",
    "    \"VMAX的上网日志业务是指...\",  # 实际应提供完整参考答案\n",
    "    \"上网日志业务主要功能包括...\",\n",
    "    \"VMAX设备日志丢失可能由...原因造成\",\n",
    "    \"解决日志存储空间不足的方法有...\",\n",
    "    \"VMAX-S与其他型号的区别主要体现在...\"\n",
    "]\n",
    "\n",
    "# 生成评估报告\n",
    "generate_evaluation_report(questions, reference_answers)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain",
   "language": "python",
   "name": "langchain"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
